#include "util.h"
#include "cache.h"
#include "data.h"
int readhead(struct datafile *df){
   struct datahead dh;
   read(df->fd,&dh,sizeof(struct datahead));
   if(dh.magic!=DATAMAGIC) return 0;
   df->nrec=dh.nrec;
   df->filesiz=dh.filesiz;
   df->tail=dh.tail;
   df->head=dh.head;
   df->areahd=dh.areahd;
   df->areatail=dh.areatail;
   df->areanum=dh.areanum;
   return 1;
}
int writehead(struct datafile *df){
   struct datahead dh;
   dh.magic=DATAMAGIC;
   //dh.nrec=df->nrec;
   dh.filesiz=df->filesiz;
   dh.tail=df->tail;
   dh.head=df->head;
   dh.areahd=df->areahd;
   dh.areatail=df->areatail;
   dh.areanum=df->areanum;
   dh.nrec=df->nrec;
   datawrite(df,(void*)&dh,sizeof(struct datahead),(uint64)0);
   printf("%s:df->nrec=%d dh.nrec=%d\n",__func__,df->nrec,dh.nrec);
   struct datahead dhd;
   dataread(df,&dhd,sizeof(struct datahead),(uint64)0);
   printf("%s:nrec=%d\n",__func__,dhd.nrec);
   return 1;
}
struct datafile *dataopen(char *name,int flag){
	if(flag==0) flag=O_CREAT|O_RDWR;
    int fd=open(name,flag,DEF_FILE_MODE);
	if(fd<0){
	    printf("data file open fail\n");
		return NULL;
	}
	struct datafile *df=calloc(1,sizeof(struct datafile));
	df->fd=fd;
	int len=strlen(name);
	df->filename=malloc(len+1);
	memcpy(df->filename,name,len);
	df->filename[len]='\0';
	struct stat sbuf;
	fstat(fd,&sbuf);
	if(sbuf.st_size>=DATAHEADSIZ&&readhead(df)){
		df->mapsiz=df->filesiz>DATAMAPSIZ?DATAMAPSIZ:df->filesiz;
		printf("%s:nrec=%d\n",__func__,df->nrec);
	}else{
	    df->filesiz=DATAHEADSIZ;
		df->nrec=0;
	    writehead(df);
		df->mapsiz=DATAHEADSIZ;
		ftruncate(fd,df->filesiz);
		printf("%s:create a new datafile\n",__func__);
	}
	if((df->mapaddr=mmap(NULL,df->mapsiz,PROT_READ|PROT_WRITE,MAP_SHARED,fd,0))==MAP_FAILED){
	    printf("can't map\n");
		df->mapaddr=NULL;
		df->mapsiz=0;
	}
	if((df->rc=rcnew(65535,512,512))==NULL){
		printf("%s:can't create cache\n",__func__);
	}
	return df;
}
int dataread(struct datafile *df,void *buf,int siz,uint64 off){
    char* ptr=buf;
	int cpsiz=0;
	if(off<df->mapsiz){
	    cpsiz=df->mapsiz-off>siz?siz:df->mapsiz-off;
		memcpy(ptr,df->mapaddr+off,cpsiz);
		ptr+=cpsiz;
		off+=cpsiz;
	}
	if(cpsiz<siz){
		lseek(df->fd,off,SEEK_SET);
		if(read(df->fd,ptr,siz-cpsiz)!=siz-cpsiz){
		    printf("%s:error=%s\n",__func__,strerror(errno));
			exit(1);
		}
	}
	return siz;
}
int datawrite(struct datafile *df,void *buf,int siz,uint64 off){
    char *ptr=buf;
	int cpsiz=0;
	if(off<df->mapsiz){
	    cpsiz=df->mapsiz-off>siz?siz:df->mapsiz-off;
		memcpy(df->mapaddr+off,ptr,cpsiz);
		ptr+=cpsiz;
		off+=cpsiz;
		//printf("%s:off=%d",__func__,off);
		//printf(" mapsiz=%d\n",df->mapsiz);
	}
	if(cpsiz<siz){
	    if(lseek(df->fd,off,SEEK_SET)<0||write(df->fd,ptr,siz-cpsiz)!=siz-cpsiz){
	        printf("%s:write error:%s\n",__func__,strerror(errno));
			return 0;
		}
	}
	return siz;
}
int sortareabylength(struct area **array,int len){
    int i,j,k,p;
	if(len<=1) return;
	i=len>>1;
	while(i>0){
	    j=i;
		k=len>>1;
		while(j<=k){
		    p=j;
			j<<=1;
		    if(j<len&&array[j-1]->length<array[j]->length) j++;
			if(array[p-1]->length>array[j-1]->length) break;
			struct area *t=array[j-1];
			array[j-1]=array[p-1];
			array[p-1]=t;
		}
		i--;
	}
	i=len;
	while(i>1){
	    i--;
		struct area *t=array[0];
		array[0]=array[i];
		array[i]=t;
		j=1;
		k=i>>1;
		while(j<=k){
		    p=j;
			j<<=1;
			if(j<i&&array[j-1]->length<array[j]->length) j++;
			if(array[j-1]->length>array[p-1]->length){//SWAP(a[p-1],a[j-1]);
			    t=array[p-1];
				array[p-1]=array[j-1];
				array[j-1]=t;
			}else break;
		}
	}
	for(i=0;i<len;i++){
		printf("%s:area %d length=%d start=%d\n",__func__,i,array[i]->length,array[i]->start);
	}
}
int sortareabystart(struct area **array,int len){
    int i,j,k,p;
	if(len<=1) return;
	i=len>>1;
	while(i>0){
	    j=i;
		k=len>>1;
		while(j<=k){
		    p=j;
			j<<=1;
		    if(j<len&&array[j-1]->start<array[j]->start) j++;
			if(array[p-1]->start>array[j-1]->start) break;
			struct area *t=array[j-1];
			array[j-1]=array[p-1];
			array[p-1]=t;
		}
		i--;
	}
	i=len;
	while(i>1){
	    i--;
		struct area *t=array[0];
		array[0]=array[i];
		array[i]=t;
		j=1;
		k=i>>1;
		while(j<=k){
		    p=j;
			j<<=1;
			if(j<i&&array[j-1]->start<array[j]->start) j++;
			if(array[j-1]->start>array[p-1]->start){
			    t=array[p-1];
				array[p-1]=array[j-1];
				array[j-1]=t;
			}
			else break;
		}
	}
}
struct area* areaalloc(struct datafile *df){
    if(df->areafreelist==NULL)
	    return malloc(sizeof(struct area));
	struct area *a=df->areafreelist;
	df->areafreelist=a->next;
	return a;
}
int areafree(struct datafile *df,struct area* a){
    a->next=df->areafreelist;
	df->areafreelist=a;
	return 1;
}
int areadefragment(struct datafile *df){
    int i,anum=0;
	struct area *t;
	struct areahead ah;
	if(df->areanum==0) return 1;
	if(df->areanum==df->arealen){
	    struct area **array=malloc(sizeof(struct area*)*df->arealen);
		struct area *a=df->arealist;
		for(i=0;i<df->areanum;i++){
			array[i]=a;
			a=a->next;
		}
	    if(df->ordertype!=ORDER_START){
		    printf("%s:going to sort %d area\n",__func__,df->areanum);
		    sortareabystart(array,df->areanum);
			int i;
			//printf("%s:areanum=%d\n",__func__,df->areanum);
			for(i=0;i<df->areanum;i++){
			    printf("%s:area %d length=%d start=%d\n",__func__,i,array[i]->length,array[i]->start);
			}
			df->ordertype=ORDER_START;
		}
		df->arealist=t=array[0];
		df->areahd=array[0]->start;
		i=1;
		while(i<df->areanum){
			while(i<df->areanum&&t->start+t->length==array[i]->start){
				t->length+=array[i]->length;
				printf("%s:free area:length=%d start=%d\n",__func__,array[i]->length,array[i]->start);
				areafree(df,array[i]);
				i++;
			}
			if(i<df->areanum){
			    t->next=array[i];
			    t=t->next;
			    i++;
			    anum++;
			}
		}
		t->next=NULL;
		df->areatail=t->start;
		df->areanext=0;
		df->arealisttail=t;
		a=df->arealist;
		struct area *pa=a;
		ah.pre=0;
		ah.next=a->next!=NULL?a->next->start:0;
		ah.siz=a->length;
		datawrite(df,&ah,sizeof(struct areahead),a->start);
		while(a->next!=NULL){
		    a=a->next;
		    ah.pre=pa->start;
			ah.next=a->next!=NULL?a->next->start:0;
			ah.siz=a->length;
			datawrite(df,&ah,sizeof(struct areahead),a->start);
			pa=a;
		}
		df->areanum=df->arealen=anum;
		free(array);
	}else{
	    int num=DATAMAXAREA>df->areanum?df->areanum:DATAMAXAREA;
	    struct areaheap *ap=areaheapnew(num);
		struct areahead temp;
		uint64 off=df->areatail,pre=0;
		uint64 newhead=0,newtail=0;
		int areanum=0,newareanum=0;
		while(areanum<df->areanum){
		    while(off!=0){
		        dataread(df,&ah,sizeof(struct areahead),off);
			    apput(ap,off,ah.siz);
			    off=ah.pre;
		    }
			for(i=0;i<ap->len;i++){
			    dataread(df,&ah,sizeof(struct areahead),ap->array[i].start);
				if(ah.pre!=0){
				    dataread(df,&temp,sizeof(struct areahead),ah.pre);
					temp.next=ah.next;
					datawrite(df,&temp,sizeof(struct areahead),ah.pre);
				}
				if(ah.next!=0){
				    dataread(df,&temp,sizeof(struct areahead),ah.next);
					temp.pre=ah.pre;
					datawrite(df,&temp,sizeof(struct areahead),ah.next);
				}
				if(df->areatail==ap->array[i].start){
				    df->areatail=ah.pre;
				}
			}
			areanum+=ap->len-1;
			if(ap->sorted!=1) apsort(ap);
			apsortall(ap);
			apmerge(ap);
			if(newhead==0){
			    newhead=ap->array[0].start;
			}
			for(i=0;i<ap->len;i++){
				temp.pre=pre;
				temp.next=i==ap->len-1?0:ap->array[i+1].start;
				temp.siz=ap->array[i].length;
				datawrite(df,&temp,sizeof(struct areahead),ap->array[i].start);
				pre=ap->array[i].start;
			}
			newareanum+=ap->len-1;
			memmove(ap->array,&ap->array[ap->len-1],sizeof(struct areaitem));
			ap->len=1;
			off=df->areatail;
		}
		df->areahd=newhead;
		df->areatail=ap->array[0].start;
		df->areanum=newareanum;
		apfree(ap);
	}
	return 1;
}
int dataloadarea(struct datafile *df){
    int num=(df->areanum>DATAMAXAREA?DATAMAXAREA:df->areanum)-df->arealen;
	int ret=num;
	uint64 off=df->areanext;
	if(off==0) off=df->areahd;
	if(off==0||num==0) return 0;
	struct areahead ah;
	memset(&ah,0,sizeof(struct areahead));
	while(num!=0&&off!=0){
	    dataread(df,&ah,sizeof(struct areahead),off);
		printf("%s:load length=%d start=%d\n",__func__,ah.siz,off);
		struct area *a=areaalloc(df);
		a->start=off;
		a->length=ah.siz;
		a->next=NULL;
		if(df->arealisttail!=NULL){
		    df->arealisttail->next=a;
			df->arealisttail=a;
		}
		else
		    df->arealist=df->arealisttail=a;
		num--;
		df->arealen++;
		off=ah.next;
	}
	df->areanext=ah.next;
	df->ordertype=ORDER_UNKOWN;
	return ret-num;
}
uint64 datagetarea(struct datafile *df,int need,int *siz){
    uint64 start;
	printf("%s:areanum=%d arealen=%d\n",__func__,df->areanum,df->arealen);
	if(df->areanum==0){
	    *siz=need;
		start=df->filesiz;
		df->filesiz+=need;
		//printf("%s:need=%d *siz=%d\n",__func__,need,*siz);
		return start;
	}
	if(df->areamiss>=DATAMAXAREAMISS){
		printf("%s:going to defragment\n",__func__);
	    areadefragment(df);
		struct area *a=df->arealist;
		struct areahead ah;
	    uint64 off=df->areahd;
	    while(a!=NULL&&off!=0){
	        dataread(df,&ah,sizeof(struct areahead),off);
		    a->start=off;
		    a->length=ah.siz;
		    a=a->next;
		    off=ah.next;
	    }
	    df->ordertype=ORDER_START;
	    df->areanext=off;
		df->areamiss=0;
	}
	if(df->arealen!=df->areanum&&df->arealen<=DATAMAXAREA/2){
	    printf("%s:load %d area\n",__func__,dataloadarea(df));
	}
	if(df->ordertype!=ORDER_LENGTH){
		printf("%s:sort area\n",__func__);
	    struct area **array=malloc(sizeof(struct area*)*df->arealen);
		struct area *a=df->arealist;
		int len=0,i;
		while(a!=NULL){
		    array[len++]=a;
			a=a->next;
		}
		sortareabylength(array,len);
		a=df->arealist=array[0];
		df->arealisttail=array[len-1];
		for(i=1;i<len;i++){
		    a->next=array[i];
			a=a->next;
		}
		a->next=NULL;
		free(array);
		df->ordertype=ORDER_LENGTH;
	}
	struct area *a=df->arealist;
	struct area *pa=a;
	printf("%s:find a fit area\n",__func__);
	while(a!=NULL&&a->length<need){
	    pa=a;
		a=a->next;
	}
	if(a!=NULL){
	    int realsiz;
		uint64 realstart;
		struct areahead ah;
		dataread(df,&ah,sizeof(struct areahead),a->start);
	    if(a->length-need>=sizeof(struct areahead)){
		    *siz=need;
			start=a->start;
			a->start+=need;
			if(df->areahd==start)  df->areahd=a->start;
			if(df->areatail==start) df->areatail=a->start;
			//if(df->areanext==start) df->areanext=a->start;
			if(ah.pre!=0){
			    struct areahead tmp;
				dataread(df,&tmp,sizeof(struct areahead),ah.pre);
				tmp.next=start;
				datawrite(df,&tmp,sizeof(struct areahead),ah.pre);
			}
			if(ah.next!=0){
			    struct areahead tmp;
				dataread(df,&tmp,sizeof(struct areahead),ah.next);
				tmp.pre=start;
				datawrite(df,&tmp,sizeof(struct areahead),ah.next);
			}
			ah.siz=a->length=a->length-need;
			datawrite(df,&ah,sizeof(struct areahead),a->start);
			if(pa==a) df->arealist=a->next;
			else pa->next=a->next;
			if(a==df->arealisttail) df->arealisttail=pa;
			areaput(df,a->start,a->length);
			areafree(df,a);
		}else{
		    *siz=a->length;
			start=a->start;
			if(df->areahd==start) df->areahd=ah.next;
			if(df->areatail==start) df->areatail=ah.pre;
			//if(df->areanext==start) df->areanext=ah.next;
			if(ah.pre!=0){
			    struct areahead tmp;
				dataread(df,&tmp,sizeof(struct areahead),ah.pre);
				tmp.next=ah.next;
				datawrite(df,&tmp,sizeof(struct areahead),ah.pre);
			}
			if(ah.next!=0){
			    struct areahead tmp;
				dataread(df,&tmp,sizeof(struct areahead),ah.next);
				tmp.pre=ah.pre;
				datawrite(df,&tmp,sizeof(struct areahead),ah.next);
			}
			if(pa==a) df->arealist=a->next;
			else pa->next=a->next;
			if(a==df->arealisttail) df->arealisttail=pa;
			areafree(df,a);
			df->arealen--;
			df->areanum--;
		}
	}else{
	    *siz=need;
		start=df->filesiz;
		df->areamiss++;
		df->filesiz+=need;
		printf("%s:alloc area at end of file\n",__func__);
	}
	return start;
}
int areaput(struct datafile *df,uint64 start,int32 len){
    if(df->filesiz==start+len){
	    df->filesiz=start;
		printf("%s:free area at end of file:start=%d\n",__func__,start);
		return 1;
	}
    struct areahead ah;
	ah.pre=df->areatail;
	ah.next=0;
	ah.siz=len;
	datawrite(df,&ah,sizeof(struct areahead),start);
	df->areanum++;
	if(df->areatail!=0){
	    dataread(df,&ah,sizeof(struct areahead),df->areatail);
		ah.next=start;
		datawrite(df,&ah,sizeof(struct areahead),df->areatail);
	}else{
	    df->areahd=start;
	}
	df->areatail=start;
	printf("%s:free area length=%d areanum=%d arealen=%d start=%d\n",__func__,len,df->areanum,df->arealen,start);
	if(df->arealen>=DATAMAXAREA||df->areanum>DATAMAXAREA){
		if(df->areanext==0) df->areanext=start;
		return 1;
	}
    struct area *a=areaalloc(df);
	a->start=start;
	a->length=len;
	if(df->ordertype!=ORDER_LENGTH){
	    printf("%s:order=%d add area to tail\n",__func__,df->ordertype);
	    a->next=NULL;
	    if(df->arealisttail==NULL) df->arealisttail=df->arealist=a;
		else{
			df->arealisttail->next=a;
			df->arealisttail=a;
		}
		df->ordertype=ORDER_UNKOWN;
	}else{
	    //printf("%s:add length=%d offset=%d\n",__func__,len,start);
		printf("%s:find a place to add area\n",__func__);
	    struct area *ap=df->arealist;
		struct area *app=ap;
		while(ap!=NULL&&ap->length<len){
		    app=ap;
			ap=ap->next;
		}
		a->next=NULL;
		if(ap==NULL&&app==NULL){
		    df->arealist=df->arealisttail=a;
		}else if(ap==NULL){
		    df->arealisttail->next=a;
			df->arealisttail=a;
		}else if(ap==app){
		    a->next=df->arealist;
			df->arealist=a;
		}else{
		    a->next=ap;
			app->next=a;
		}
	}
	df->arealen++;
	return 1;
}
#define FILLRECORD(dr,size,key,keysiz,val,valsiz,pre_rec,next_rec)\
    do{\
	     (dr)->siz=size;\
	     (dr)->rsiz=(keysiz)+(valsiz)+sizeof(struct datarecord);\
		 (dr)->pre=pre_rec;\
		 (dr)->next=next_rec;\
		 memcpy((dr)+1,key,keysiz);\
		 memcpy((char*)((dr)+1)+keysiz,val,valsiz);\
	}while(0)
uint64 dataput(struct datafile *df,char *key,int ksiz,char *val,int vsiz){
    char buf[BLOCKSIZ];
	struct datarecord *dr=(struct datarecord*)buf;
	int recsiz=sizeof(struct datarecord)+ksiz+vsiz;
	if(recsiz>BLOCKSIZ)
	    dr=malloc(recsiz);
	uint64 start;
	int len;
	start=datagetarea(df,recsiz,&len);
	printf("%s:nrec=%d rec len=%d recsiz=%d start=%d\n",__func__,df->nrec,len,recsiz,start);
	FILLRECORD(dr,len,key,ksiz,val,vsiz,df->tail,0);
	datawrite(df,(char*)dr,recsiz,start);
	if(df->tail!=0){
	    dataread(df,buf,sizeof(struct datarecord),df->tail);
		((struct datarecord*)buf)->next=start;
		datawrite(df,buf,sizeof(struct datarecord),df->tail);
		df->tail=start;
	}else{
	    df->head=df->tail=start;
	}
	df->nrec++;
	if((char*)dr!=buf) free(dr);
	return start;
}
uint64 dataputrec(struct datafile *df,char *data,int dsiz){
    char buf[BLOCKSIZ];
	struct datarecord *dr=(struct datarecord*)buf;
	int recsiz=sizeof(struct datarecord)+dsiz;
	if(recsiz>BLOCKSIZ)
	    dr=malloc(recsiz);
	uint64 start;
	int len;
	start=datagetarea(df,recsiz,&len);
	printf("%s:nrec=%d rec len=%d recsiz=%d dsiz=%d start=%d\n",__func__,df->nrec,len,recsiz,dsiz,start);
	dr->siz=len;
	dr->rsiz=recsiz;
    dr->pre=df->tail;
	dr->next=0;
	memcpy(dr+1,data,dsiz);
	datawrite(df,(char*)dr,recsiz,start);
	if(df->tail!=0){
	    dataread(df,buf,sizeof(struct datarecord),df->tail);
		((struct datarecord*)buf)->next=start;
		datawrite(df,buf,sizeof(struct datarecord),df->tail);
		df->tail=start;
	}else{
	    df->head=df->tail=start;
	}
	df->nrec++;
	if((char*)dr!=buf) free(dr);
	return start;
}
int dataget(struct datafile *df,uint64 offset,struct bstring **val){
	struct datarecord *data;
	int recsiz,pos;
    if(val==NULL||offset>df->filesiz||offset<DATAHEADSIZ) return 0;
	struct hashnode *hn=getrecnode(df->rc,(char*)&offset,sizeof(uint64),&pos);
	if(hn!=NULL){
		printf("%s:cache hit\n",__func__);
		if(*val==NULL) *val=strnew(hn->vsiz);
		else if((*val)->max<hn->vsiz) stradjust(*val,hn->vsiz);
		rcgetval(df->rc,hn,pos,(*val)->array,hn->vsiz);
		(*val)->siz=hn->vsiz;
		return 1;
	}
	if(df->mapsiz>=offset+sizeof(struct datarecord)){
	    data=(struct datarecord*)((char*)df->mapaddr+offset);
		recsiz=data->rsiz-sizeof(struct datarecord);
		if(*val==NULL) *val=strnew(recsiz);
	    else if((*val)->max<recsiz)
	        stradjust(*val,recsiz);
		int siz=df->mapsiz-offset-sizeof(struct datarecord);
		if(siz>0){
		    int cpsiz=siz>recsiz?recsiz:siz;
			memcpy((*val)->array,data+1,cpsiz);
			if(cpsiz<recsiz){
			    seekread(df->fd,(*val)->array+cpsiz,recsiz-cpsiz,offset+sizeof(struct datarecord)+cpsiz);
			}
		}else{
		    seekread(df->fd,(*val)->array,recsiz,offset+sizeof(struct datarecord));
		}
		(*val)->siz=recsiz;
	}else{
	    char buf[BLOCKSIZ];
		int rdsiz=sizeof(buf)>df->filesiz-offset?df->filesiz-offset:sizeof(buf);
	    seekread(df->fd,buf,rdsiz,offset);
	    data=(struct datarecord*)buf;
	    recsiz=data->rsiz-sizeof(struct datarecord);
		printf("%s:record siz=%d\n",__func__,data->rsiz);
		if(*val==NULL) *val=strnew(recsiz);
	    else if((*val)->max<recsiz)
	        stradjust(*val,recsiz);
	    if(recsiz>BLOCKSIZ-sizeof(struct datarecord)){    
		    memcpy((*val)->array,data+1,BLOCKSIZ-sizeof(struct datarecord));
		    read(df->fd,(*val)->array+BLOCKSIZ-sizeof(struct datarecord),data->rsiz-BLOCKSIZ+sizeof(struct datarecord));
	    }else
		    memcpy((*val)->array,data+1,recsiz);
		(*val)->siz=recsiz;
	}
	printf("%s:recsiz=%d\n",__func__,(*val)->siz);
	rcput(df->rc,(char*)&offset,sizeof(uint64),(*val)->array,(*val)->siz);
	return 1;
}
int dataerase(struct datafile *df,uint64 offset){
    struct datarecord data;
	if(offset>=df->filesiz||offset<DATAHEADSIZ) return 0;
	rcremove(df->rc,(char*)&offset,sizeof(uint64));
	dataread(df,(char*)&data,sizeof(struct datarecord),offset);
	if(offset==df->tail) df->tail=data.pre;
	if(offset==df->head) df->head=data.next;
	if(offset==df->areanext) df->areanext=data.next;
	areaput(df,offset,data.siz);
	df->nrec--;
}
int dataiter_init(struct datafile *df,uint64 offset){
    if(offset==DF_ITER_START){
	    df->iter=df->head;
	}else if(offset==DF_ITER_LAST){
	    df->iter=df->tail;
	}else df->iter=offset;
	return 1;
}
int dataiter_next(struct datafile *df,struct bstring **key,struct bstring **val){
    struct datarecord *data;
	int recsiz;
	if(val==NULL||df->iter>=df->filesiz||df->iter<DATAHEADSIZ) return 0;
	if(df->mapsiz>=df->iter+sizeof(struct datarecord)){
	    data=(struct datarecord*)((char*)df->mapaddr+df->iter);
		recsiz=data->rsiz-sizeof(struct datarecord);
		if(*val==NULL) *val=strnew(recsiz);
		else if((*val)->max<recsiz) stradjust(*val,recsiz);
		int siz=df->mapsiz-df->iter-sizeof(struct datarecord);
		if(siz>0){
		    int cpsiz=siz>recsiz?recsiz:siz;
			memcpy((*val)->array,(char*)df->mapaddr+df->iter+sizeof(struct datarecord),cpsiz);
			if(siz<recsiz){
			    seekread(df->fd,(*val)->array+cpsiz,recsiz-cpsiz,df->iter+sizeof(struct datarecord)+cpsiz);
			}
		}else{
		    seekread(df->fd,(char*)(*val)->array,recsiz,df->iter+sizeof(struct datarecord));
		}	
	}else{
	    char buf[BLOCKSIZ];
		int rdsiz=sizeof(buf)>df->filesiz-df->iter?df->filesiz-df->iter:sizeof(buf);
		seekread(df->fd,buf,rdsiz,df->iter);
		data=(struct datarecord*)buf;
		recsiz=data->rsiz;
		if(*val==NULL) *val=strnew(recsiz);
		else if((*val)->max<recsiz) stradjust(*val,recsiz);
		int cpsiz=rdsiz-sizeof(struct datarecord)>recsiz?recsiz:rdsiz-sizeof(struct datarecord);
		memcpy((*val)->array,buf+sizeof(struct datarecord),cpsiz);
		if(cpsiz<recsiz){
		    read(df->fd,(*val)->array+cpsiz,recsiz-cpsiz);
		}
	}
	if(key!=NULL){
		if(*key==NULL) *key=strcreate((char*)&df->iter,sizeof(df->iter));
		else{
			if((*key)->max<sizeof(df->iter)) stradjust(*key,sizeof(df->iter));
			memcpy((*key)->array,&df->iter,sizeof(uint64));
			(*key)->siz=sizeof(df->iter);
		}
	}
	df->iter=data->next;
	(*val)->siz=data->rsiz-sizeof(struct datarecord);
	return 1;
}
int dataiter_pre(struct datafile *df,struct bstring **key,struct bstring **val){
    if(val==NULL||df->iter<DATAHEADSIZ||df->iter>=df->filesiz) return 0;
	struct datarecord *data;
	int recsiz;
	if(df->mapsiz>=df->iter+sizeof(struct datarecord)){
	    data=(struct datarecord*)((char*)df->mapaddr+df->iter);
		recsiz=data->rsiz-sizeof(struct datarecord);
		if(*val==NULL) *val=strnew(recsiz);
		else if((*val)->max<recsiz) stradjust(*val,recsiz);
		int siz=df->mapsiz-df->iter-sizeof(struct datarecord);
		if(siz>0){
		    int cpsiz=siz>recsiz?recsiz:siz;
			memcpy((*val)->array,(char*)df->mapaddr+df->iter+sizeof(struct datarecord),cpsiz);
			if(siz<recsiz){
			    seekread(df->fd,(*val)->array+cpsiz,recsiz-cpsiz,df->iter+sizeof(struct datarecord)+cpsiz);
			}
		}else{
		    seekread(df->fd,(*val)->array,recsiz,df->iter+sizeof(struct datarecord));
		}
		//df->iter=data->pre;
		//(*val)->siz=recsiz;
	}else{
	    char buf[BLOCKSIZ];
		int rdsiz=df->filesiz-df->iter>sizeof(buf)?sizeof(buf):df->filesiz-df->iter;
		seekread(df->fd,buf,rdsiz,df->iter);
		data=(struct datarecord*)buf;
		recsiz=data->rsiz-sizeof(struct datarecord);
		if(*val==NULL) *val=strnew(recsiz);
		else if((*val)->max<recsiz) stradjust(*val,recsiz);
		int cpsiz=rdsiz-sizeof(struct datarecord)>recsiz?recsiz:rdsiz-sizeof(struct datarecord);
		memcpy((*val)->array,buf+sizeof(struct datarecord),cpsiz);
		if(cpsiz<recsiz){
		    read(df->fd,(*val)->array+cpsiz,recsiz-cpsiz);
		}
		//df->iter=data->pre;
		//(*val)->siz=recsiz;
	}
	if(key!=NULL){
		if(*key==NULL) *key=strcreate((char*)&df->iter,sizeof(uint64));
		else{
			if((*key)->max<sizeof(df->iter)) stradjust(*key,sizeof(df->iter));
			memcpy((*key)->array,&df->iter,sizeof(df->iter));
			(*key)->siz=sizeof(df->iter);
		}
	}
	df->iter=data->pre;
	(*val)->siz=data->rsiz-sizeof(struct datarecord);
	return 1;
}
int dataclose(struct datafile *df){
	if(df->areanum>=(df->nrec>>1)){
		printf("%s:defragment df->areanum=%d\n",__func__,df->areanum);
	    areadefragment(df);
		uint64 off=df->areatail;
		struct areahead ah;
		dataread(df,&ah,sizeof(struct areahead),off);
		while(off!=0&&off+ah.siz==df->filesiz){
		    df->filesiz-=ah.siz;
			off=ah.pre;
		}
		df->areatail=off;
	}
	char *ptr=df->mapaddr;
	writehead(df);
	if(df->mapaddr!=NULL){
	    msync(df->mapaddr,df->mapsiz,MS_SYNC);
	    munmap(df->mapaddr,df->mapsiz);
		printf("%s:map sync\n",__func__);
	}
	ftruncate(df->fd,df->filesiz);
	fsync(df->fd);
	close(df->fd);
	free(df->filename);
	struct area *a=df->arealist;
	printf("%s:free arealist\n",__func__);
	while(a!=NULL){
	    struct area *t=a->next;
		free(a);
		a=t;
	}
	a=df->areafreelist;
	printf("%s:free areafreelist\n",__func__);
	while(a!=NULL){
	    struct area *t=a->next;
		free(a);
		a=t;
	}
	rcfree(df->rc);
	printf("%s:free reccache ok\n",__func__);
	free(df);
	return 1;
}
/* 
void itoa(int num,char *key,int siz){
   int i;
   for(i=siz-2;i>=0;i--){
      key[i]=num%10+'0';
	  num/=10;
   }
   key[siz-1]='\0';
}

#define KNUM 100
int main(){
	char buf[23];
	int i,k;
	uint64 offs[KNUM];
	memset(offs,0,sizeof(offs));
	struct datafile *df=dataopen("test.dat",O_CREAT|O_RDWR|O_TRUNC);
	if(df==NULL){
	    printf("%s:open fail\n",__func__);
		exit(1);
	}
	srand(1);
	for(i=0;i<KNUM;i++){
	    k=rand()%KNUM;
		if(offs[k]!=0) continue;
		itoa(k,buf,sizeof(buf));
		if(!(offs[k]=dataput(df,buf,sizeof(buf)-1,buf,sizeof(buf)-1))){
		    printf("%s:put %d fail\n",__func__,i);
			exit(1);
		}else printf("%s:record %d key=%d offset=%d\n",__func__,i,k,offs[k]);
	}
	// dataiter_init(df,DF_ITER_LAST);
	// struct bstring *key=NULL,*val=NULL;
	// while(dataiter_pre(df,&key,&val)&&key!=NULL){
	    // printf("%s:data=%s offset=%d\n",__func__,val->array,*((uint64*)key->array));
	// }
	// if(key!=NULL) strdel(key);
	// if(val!=NULL) strdel(val);
	struct bstring *val=NULL;
	for(i=0;i<KNUM;i++){
		if(offs[i]!=0&&dataget(df,offs[i],&val)&&val!=NULL){
			printf("%s:get %d val=%s off=%d\n",__func__,i,val->array,offs[i]);
		}
	}
	if(val!=NULL) strdel(val);
	for(i=0;i<KNUM/2;i++){
	    if(offs[i]!=0){
			if(!dataerase(df,offs[i])){
				printf("%s:erase %d fail!off=%d\n",__func__,i,offs[i]);
				exit(1);
			}else printf("%s:erase %d ok\n",__func__,i);
			offs[i]=0;
		}
	}
	printf("%s:nrec=%d\n",__func__,df->nrec);
	for(i=0;i<KNUM;i++){
	    k=rand()%KNUM;
		if(offs[k]!=0) continue;
		itoa(k,buf,sizeof(buf));
		if((offs[k]=dataput(df,buf,sizeof(buf)-1,buf,sizeof(buf)-1))==0){
		    printf("%s:put %d fail\n",__func__,i);
			exit(1);
		}else printf("%s:record %d key=%d offset=%d\n",__func__,i,k,offs[k]);
	}
	val=NULL;
	for(i=0;i<KNUM;i++){
		if(offs[i]!=0&&dataget(df,offs[i],&val)&&val!=NULL){
			printf("%s:get record %d val=%s off=%d\n",__func__,i,val->array,offs[i]);
		}
	}
	if(val!=NULL) strdel(val);
	dataclose(df);
} */
/*
int main(){
    struct datafile *df=dataopen("test.dat",O_CREAT|O_RDWR);
	if(df==NULL){
	    printf("open fail\n");
		return false;
	}
	FILE *fp=fopen("logtab.txt","r");
	char buf[128],*key,*val;
	int num=0,ksiz,vsiz;
	struct timeval beg,end;
	fgets(buf,sizeof(buf),fp);
	gettimeofday(&beg,NULL);
	while(num<10&&fgets(buf,sizeof(buf),fp)!=NULL){
		char *pos;
		int len=strlen(buf);
		if((pos=memchr(buf,'\t',len))!=NULL){
		    key=buf;
			ksiz=pos-buf;
			val=pos+1;
			vsiz=len-ksiz-1;
			//printf("key=%s len=%d val=%s vlen=%d\n",buf,key.size,(char*)data.data,data.size);
			if(!dataput(df,key,ksiz,val,vsiz)){
			    printf("put fail\n");
				exit(1);
			}
			num++;
		}else{
		    printf("bad record:%s\n",buf);
		}
	}
	gettimeofday(&end,NULL);
	printf("insert %d records cost %f sec\n",num,end.tv_sec-beg.tv_sec+(end.tv_usec-beg.tv_usec)/1000000.0);
	dataclose(df);
}*/